package InformationExtraction;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Collections;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import DB.*;



public class DataExtraction {

	private Data data;
	private Table bornIn;
	private Table profession;
	private Table moviesByYear;
	private Table directors;
	private Table capitals;
	private Table albums;
	private Table artistsAlbums;
	private Vector<String> bornInPersons;
	private Vector<String> profPersons;
	private Vector<String> allAlbums; 
	private Vector<String> allDirectors;
	private Vector<String> allCountries;
	private Vector<String> allArtists;
	private Vector<String> allDirectedMovies;
	private Vector<String> specifictProf;
	
/*
 * Constructor
 */
	
	public DataExtraction(){
		this.data = new Data();
		createTables();
		bornInPersons = new  Vector<String>();
		profPersons = new Vector<String>();
		allAlbums = new Vector<String>();
		allDirectors = new Vector<String>();
		allCountries = new Vector<String>();
		allArtists = new Vector<String>();
		allDirectedMovies = new Vector<String>();
		specifictProf = new Vector<String>();
		specifictProf.add("scientist");
		specifictProf.add("philosopher");
		specifictProf.add("director");
		specifictProf.add("actor");
		specifictProf.add("writer");
		specifictProf.add("doctor");
		specifictProf.add("artist");
		specifictProf.add("composer");
		specifictProf.add("dancer");
		specifictProf.add("singer");
		specifictProf.add("explorer");
		specifictProf.add("producer");
		specifictProf.add("mathematician");

		read("full_pos.txt");
//		read("TestSet.txt");
		data.printToFile("data.txt");
	}

/*
 * Create all seven relations with there name, fields and keys
 */
	
	private void createTables() {
		bornIn = defineTable("BornIn", "Person", "Year", 1, 2.27);
		data.addTbale(bornIn);
		profession = defineTable("Profession", "Person", "Prof", 2, 1.3);
		data.addTbale(profession);
		moviesByYear = defineTable("MoviesByYear", "Movie", "Year", 2, 2.61);
		data.addTbale(moviesByYear);
		directors = defineTable("MoviesDirectors", "Movie", "Director", 2, 0.85);
		data.addTbale(directors);
		capitals = defineTable("Capitals", "Country", "City", 1, 1.65);
		data.addTbale(capitals);
		albums = defineTable("Albums", "Album", "Year", 1, 1.15);
		data.addTbale(albums);
		artistsAlbums = defineTable("ArtistsAlbums", "Artist", "Album", 2, 2.3815);
		data.addTbale(artistsAlbums);
	}

/*
 *  Extracting data from the input file. The text is read line by line, Every line
 *  is a value needed to be processed by the different heuristics and eventually
 *  pieces of data get in the tables mentioned above. 
 */
	
	public void read(String filename) {
		try{
			FileInputStream fstream = new FileInputStream(filename);
			// Get the object of DataInputStream
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String strLine="";
			//Read File Line By Line
			String currValue = "";
			while ((strLine = br.readLine()) != null)   {

				if (strLine.isEmpty()){
					currValue = "";
				}
				
				else if (currValue.isEmpty()){
					currValue = strLine;
				}
				else{					
					findPerson(currValue, strLine);	
					findMovies(currValue, strLine);
					findAlbums(currValue,strLine);
					findCapital(currValue, strLine);
					findProfessionDirectly(currValue, strLine);
					heuristicByYear(currValue, strLine);
					findByAward(currValue, strLine);
					findMoviesFast(currValue, strLine);
					findPersonDiffrent(currValue, strLine);
					findCapitalByCountry(currValue, strLine);
				}
			}			
			//Close the input stream
			in.close();

		}catch (Exception e){//Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}


//*********************************//
/*           Heuristics		       */
//*********************************//	

/*
 *  Heuristic for the Table "bornIn" - by finding year and the word "born"
 *  close together and the year less then 2012
 */
	private void findPerson(String value,String strLine) {

		double prob = 0.99;
		String year = "(\\d\\d\\d\\d)/CD";
		String born = "born/VBN";
		Pattern yearPattern = Pattern.compile(year);
		Pattern bornPattern = Pattern.compile(born);
		Matcher yearMatcher = yearPattern.matcher(strLine);
		Matcher bornMatcher = bornPattern.matcher(strLine);
		if (yearMatcher.find()){ 
			if (bornMatcher.find()) {		
				if (((yearMatcher.start() - bornMatcher.end()) < 50) && (yearMatcher.start() > bornMatcher.end())){
					if(Integer.parseInt(yearMatcher.group(1)) < 2012){
						findProfession(value, strLine);
						addTuple(bornIn, value, yearMatcher.group(1), prob);		
						if (!bornInPersons.contains(value))
							bornInPersons.add(value);
					}
				}
			}
		}
	}
	
/*
 *  Heuristic for the Table "bornIn" - in the first sentence the is brackets with 
 *  a full date
 */
	
	private void findPersonDiffrent(String value, String strLine) {
		double prob = 0.8;
		if (checkRealWord(value)){
			if (strLine.indexOf("./.") > 0)
				strLine = strLine.substring(0, strLine.indexOf("./."));
			String brk = "-LRB-/-LRB-.*-RRB-/-RRB-";
			String year = "(\\d\\d\\d\\d)/CD";
			String day = "\\d\\d/CD";
			Pattern brkPattern = Pattern.compile(brk);
			Pattern yearPattern = Pattern.compile(year);
			Pattern dayPattern = Pattern.compile(day);
			Matcher brkMatcher = brkPattern.matcher(strLine);
			Matcher yearMatcher = yearPattern.matcher(strLine);
			Matcher dayMatcher = dayPattern.matcher(strLine);
			if (yearMatcher.find() && brkMatcher.find() && dayMatcher.find()){ 
				findProfession(value, strLine);
				addTuple(bornIn, value, yearMatcher.group(1), prob);
				if (!bornInPersons.contains(value))
					bornInPersons.add(value);
			}
		}
	}

/*
 * Heuristic for the Table "profession" - by finding specific 
 * professions we defined 
 */
	
	private void findProfessionDirectly(String value, String strLine) {
		double prob = 0.57;
		for (int i = 0 ; i < specifictProf.size(); i++){		
			Pattern specificProfPattern = Pattern.compile(specifictProf.elementAt(i));
			Matcher specificStrMatcher = specificProfPattern.matcher(strLine);
			if (specificStrMatcher.find()){
				strLine = strLine.substring(specificStrMatcher.end()-1);		
				int firstLineEnd = strLine.indexOf("./.");
				if (firstLineEnd > 0)
					strLine = strLine.substring(0, firstLineEnd);
				String name = findNameNoSpace(strLine);
				if (name.length() > 0 && name.length() < 20 && checkRealWord(name)){
					addTuple(profession, name, specifictProf.elementAt(i), prob);	
				}
			}
		}
	}

/*
 * Heuristic for the Table "moviesByYear" and the Table "bornIn" -
 * by looking if there is a mention of the Academy award
 */
	
	private void findByAward(String value, String strLine){
		double prob = 0.48;
		String year =findYear(strLine);
		if (year!=null){
		String award = "Academy/NNP Award/NNP";
			Pattern awardPattern = Pattern.compile(award);
			Matcher awardMatcher = awardPattern.matcher(strLine);
			if (awardMatcher.find()){
				addTuple(this.moviesByYear, value, year, prob);
				if (value.indexOf("movie") < 0){
				addTuple(this.bornIn, value, year, prob);
				if (!bornInPersons.contains(value))
					bornInPersons.add(value);
				}				
			}
		}
	}

/*
 * Heuristic for the Table "profession" - locating values of people 
 * and searching for there profession 
 */
	
	private void findProfession(String value, String strLine) {
		String startStr = "/VB.?\\s+\\w+/DT\\s+";
		Pattern startStrPattern = Pattern.compile(startStr);
		Matcher startStrMatcher = startStrPattern.matcher(strLine);
		if (startStrMatcher.find()){
			strLine = strLine.substring(startStrMatcher.end()-1);
		}
		int firstLineEnd = strLine.indexOf("./.");
		if (firstLineEnd > 0)
			strLine = strLine.substring(0, firstLineEnd);
		firstLineEnd = strLine.indexOf("who");
		if (firstLineEnd > 0)
			strLine = strLine.substring(0, firstLineEnd);
		String prof = "\\s+(\\w+)/NN";
		Pattern profPattern = Pattern.compile(prof);
		Matcher profMatcher = profPattern.matcher(strLine);
		Vector<String> profs = new Vector<String>();

		while (profMatcher.find()){
			profs.add(profMatcher.group(1));
		}
		if (!profs.isEmpty()){
			addProf(value, strLine, profs);		
		}
	}

/*
 * Heuristic for the Table "profession" - continuation
 * from the function above 
 */
	
	private void addProf(String value, String strLine, Vector<String> profs) {
		double prob = 0.73;
		int startIndex = strLine.indexOf(profs.elementAt(0));
		int endIndex = startIndex + profs.elementAt(0).length();
		if (profs.size() > 0){
			endIndex = strLine.indexOf(profs.elementAt(profs.size() - 1), startIndex) + profs.elementAt(profs.size() - 1).length();
		}		
		String tmpStr = strLine.substring(startIndex, endIndex);
		tmpStr = tmpStr.replaceAll("/\\w+", "");
		tmpStr = tmpStr.replaceAll(",/,", ",");
		tmpStr = tmpStr.replaceAll("-LRB-/-LRB-", "(");
		tmpStr = tmpStr.replaceAll("-RRB-/-RRB-", ")");
		String [] splittedProf = tmpStr.split("(,|and)");
		for ( int  i =0 ; i<splittedProf.length; i++){
			splittedProf[i] = splittedProf[i].trim();
			if (splittedProf[i].length()>0 && checkRealWord(splittedProf[i])){
				checkProf(splittedProf[i], value, strLine);
				addTuple(profession, value, splittedProf[i], prob);	
				if (!profPersons.contains(splittedProf[i]))
					profPersons.add(splittedProf[i]);				
			}
		}
	}
	
/*
 * Heuristic for the Table "moviesByYear" - catching the value with
 * the word "movie" in there title
 */
	
	private void findMoviesFast(String value, String strLine) {
		double prob = 0.9;
		String movie = "(movie)";
		Pattern moviePattern = Pattern.compile(movie);
		Matcher movieMatcher = moviePattern.matcher(value);
		if (movieMatcher.find()){
			String year = findYear(strLine);
			if (year != null){
				if(value.indexOf("(") > 0){
					value = value.substring(0, value.indexOf("("));
				}
				if(value.substring(value.length() - 1).equals(" "))
					value = value.substring(0, value.length() - 1);
				addTuple(this.moviesByYear, value, year, prob);
			}
		}
	}

/*
 * Heuristic divided to two - in the first part find all the value 
 * involved with years, and in the second, checks if it concerns
 * movies or albums  
 */
	
	private void heuristicByYear(String value, String strLine){
		Pattern yearPattern = Pattern.compile( "(\\d\\d\\d\\d)/CD");
		Matcher yearMatcher = yearPattern.matcher(strLine);
		int yearIndex = 0;
		String year = "";
		while (yearMatcher.find()){
			yearIndex = yearMatcher.start();
			year = yearMatcher.group(1);
			Pattern startStrPattern = Pattern.compile("movies");
			Matcher startStrMatcher = startStrPattern.matcher(strLine);
			if (startStrMatcher.find() && yearIndex > 50)	
				findMovieByYear(strLine.substring(yearIndex - 50 ,yearIndex) ,year);		
			Pattern recordStrPattern = Pattern.compile("Record/NNP of/IN the/DT Year/NN");
			Matcher recordStrMatcher = recordStrPattern.matcher(strLine);
			if (recordStrMatcher.find() && yearIndex < recordStrMatcher.start()){	
				findAlbumByYear(value, strLine.substring(recordStrMatcher.start()+10) ,year);
			}
		}
	}

/*
 * Heuristic for the Table "moviesByYear" - continuation
 * from the function above 
 */
	
	private void findMovieByYear(String strLine, String year) {
		double prob = 0.4;
		String name = findFullName(strLine);
		if (name.length() > 0){
			addTuple(this.moviesByYear, name, year, prob);			
		}			
	}
	
/*
 * Heuristic for the Table "albums" and the Table
 * "artistsAlbums" - continuation from the function above 
 */
	
	private void findAlbumByYear(String value, String strLine, String year) {
		double yearProb = 0.25;
		double bandProb = 0.999;
		int firstLineEnd = strLine.indexOf("./.");
		if (firstLineEnd > 0)
			strLine = strLine.substring(0, firstLineEnd);
		String name = findFullName(strLine);
		if (name.length() > 0){
			addTuple(this.albums, name, year, yearProb);
			addTuple(this.artistsAlbums, value, name, bandProb);
			if (!allAlbums.contains(name))
				allAlbums.add(name);
			if (!allArtists.contains(value))
				allArtists.add(value);
		}
	}

/*
 * Heuristic for the Table "artistsAlbums" - in case we found a value 
 * of some singer, check if it mentions an album he made
 */
	
	private void checkProf(String prof, String value, String strLine) {
		if (prof.equals("singer")){
			String albumName = findFullName(strLine);
			if (albumName != null && albumName.length() > 0 ){
				addTuple(this.artistsAlbums, value, albumName, 0.2);
				if (!allAlbums.contains(albumName))
					allAlbums.add(albumName);
				if (!allArtists.contains(value))
					allArtists.add(value);
			}
		}
	}	

/*
 * Heuristic for the Table "artistsAlbums" - special case with
 * the band "Kiss" which it's name is a verb and not tagged with "NNP"
 */
	
	private void findAlbumsDirectly(String value, String strLine) {
		double prob = 0.67;
		String Kiss = "Kiss";
		Pattern KissPattern = Pattern.compile(Kiss);
		Matcher KissMatcher = KissPattern.matcher(strLine);
		if (KissMatcher.find()){		
			addTuple(artistsAlbums, Kiss, value, prob);
			if(!allArtists.contains("Kiss"))
				allArtists.add("Kiss");
		}
	}
	
/*
 * Heuristic for the Table "albums" - catching the value with
 * the word "album" in there title 
 */
	private void findAlbums(String value, String strLine) {
		double prob = 0.9;
		String album = "(album)";
		Pattern albumPattern = Pattern.compile(album);
		Matcher albumMatcher = albumPattern.matcher(value);
		if (albumMatcher.find()){
			String year = findYear(strLine);
			if (year != null){
				if(value.indexOf("(") > 0){
					value = value.substring(0, value.indexOf("("));
				}
				addTuple(this.albums, value, year, prob);
				allAlbums.add(value);
			}
			byOrOfFindAlbumsArtist(value, strLine);
			findAlbumsDirectly(value, strLine);
		}
	}

/*
 * Heuristic for the Table "artistsAlbums" - by the regular expression
 * "\\s+(by|of)/IN" which indicates in case the value concerns some album
 * made by/of some artist.
 */
	
	private void byOrOfFindAlbumsArtist(String value, String strLine) {
		double prob = 0.5125;
		String byOrOf = "\\s+(by|of)/IN";
		Pattern byOrOfPattern = Pattern.compile(byOrOf);
		Matcher byOrOfMatcher = byOrOfPattern.matcher(strLine);
		if (byOrOfMatcher.find()){
			String newLine = strLine.substring(byOrOfMatcher.end());
			String name = findName(newLine);
			if(value.indexOf("(") > 0){
				value = value.substring(0, value.indexOf("("));
			}
			if (name != null && name != ""){
				addTuple(artistsAlbums, name, value, prob);
				if(!allArtists.contains(name))
					allArtists.add(name);
			}
		}
	}

/*
 * Heuristic for the Table "moviesByYear" and the Table "directors"
 * in case one of the words movie or film shows and one of the words
 * directed or director appear
 */
	
	private void findMovies(String value,String strLine) {
		double prob = 0.83;
		double probDir = 0.85;
		String movie = "(movie|film)";
		String director = "(directed|director)";
		Pattern moviePattern = Pattern.compile(movie);
		Pattern directorPattern = Pattern.compile(director);	
		Matcher movieMatcher = moviePattern.matcher(strLine);
		Matcher directorMatcher = directorPattern.matcher(strLine);
		if (movieMatcher.find()){
			if(movieMatcher.start() < 300){	
				String name = findFullName(strLine);
				if (name != null && name.length() > 0){
					String year = findYear(strLine);
					if (year != null && Integer.parseInt(year) > 1930 && Integer.parseInt(year) < 2012)
						addTuple(this.moviesByYear, name, year, prob);
					if (directorMatcher.find()){
						String directorName = findDirector(strLine);
						if (directorName != null && directorName.length() > 0 &&
								checkRealWord(directorName)){							
							addTuple(this.directors, name, directorName, probDir);
							if (!allDirectedMovies.contains(name))
								allDirectedMovies.add(name);
							if (!allDirectors.contains(directorName))
								allDirectors.add(directorName);
						}
					}
				}
			}
		}			
	}
	
/*
 * Heuristic for the Table "capitals" - if the value is of a 
 * capital the first sentence in the value will be "**** is the 
 * capital of..."
 */
	
	private void findCapital(String value,String strLine){
		double prob = 0.73;
		String newLine = "";
		String capital = "is/VBZ\\s+the/DT\\s+capital";
		Vector<String> ans = new Vector<String>();
		Pattern capitalStrPattern = Pattern.compile(capital);		
		Matcher capitalStrMatcher = capitalStrPattern.matcher(strLine);
		if(capitalStrMatcher.find()){
			newLine = strLine.substring(capitalStrMatcher.end());
			if (strLine.indexOf("./.") > 0)
				newLine = newLine.substring(0, newLine.indexOf("./."));
			String country = "(\\w+)/NNP";
			Pattern countryStrPattern = Pattern.compile(country);
			Matcher countryStrMatcher = countryStrPattern.matcher(newLine);			
			while (countryStrMatcher.find()) 
				ans.add(countryStrMatcher.group(1));					
			if (ans.size()>0){
				country = ans.elementAt(0);
				for (int i = 1; i < ans.size(); i++)
					country = country + " " + ans.elementAt(i);
				addTuple(this.capitals, country, value, prob);
				if (!allCountries.contains(country))
					allCountries.add(country);
			}

		}
	}	

/*
 * Heuristic for the Table "capitals" - if the value is of a 
 * country, it probably has a capital mentioned in the value:
 *  "its capital city is.."
 */
	private void findCapitalByCountry(String value, String strLine) {
		double prob = 0.92;
		String lookFor = "capital/NN city/NN is/VBZ";
		Pattern lookForPattern = Pattern.compile(lookFor);
		Matcher lookForMatcher = lookForPattern.matcher(strLine);
		if (lookForMatcher.find()){
			String city = findName(strLine.substring(lookForMatcher.start()));
			if (city != null && city.length() > 0){
				addTuple(this.capitals, value, city, prob);
				if (!allCountries.contains(value))
					allCountries.add(value);
			}				
		}
	}					
	
	
//*********************************//
/*         Help Function		   */
//*********************************//
	
/*
 * Initializing a Table
 */
	private Table defineTable(String tableName, String field1, String field2, int key, double prob){
		Vector<String> titles = new Vector<String>();
		titles.add(field1);
		titles.add(field2);
		Vector<String> keys = new Vector<String>();
		keys.add(field1);
		if(key > 1){
			keys.add(field2);
		}
		return new Table(tableName, keys, titles, prob);
	}

/*
 * Add a tuple to the Table
 */
	private void addTuple(Table table, String value1, String value2, double prob) {
		Vector<String> tuple = new Vector<String>();
		tuple.add(value1);
		tuple.add(value2);
		table.addTuple(tuple, prob);
	}

/*
 * Checks proper names - the word does not include dotes, hyphens
 * commas and digits 
 */
	
	private boolean checkRealWord(String word) {
		String digit = "(\\d)";
		Pattern digitPattern = Pattern.compile(digit);
		Matcher digitMatcher = digitPattern.matcher(word);
		if (!digitMatcher.find())
			if (word.indexOf(".") < 0 && word.indexOf("-") < 0 &&
					word.indexOf(",") < 0 &&
					  	word.indexOf(")") < 0 && word.indexOf("(") < 0)
				return true;		
		return false;
	}

/*
 * 	Finds the first name -> proper noun, which appears in the line  
 */
	
	private String findName(String strLine) {
		String ans = "";
		String[] nnps = strLine.split(" ");
		String dirName = "(\\w+)/NNP";
		Pattern dirNamePattern = Pattern.compile(dirName);
		Matcher nnpMatcher;
		for(int i = 0; i < nnps.length; i++){
			nnpMatcher = dirNamePattern.matcher(nnps[i]);
			if (nnpMatcher.find()){
				ans = ans + nnpMatcher.group(1);
				for(int j = i + 1; j < nnps.length; j++){
					nnpMatcher = dirNamePattern.matcher(nnps[j]);
					if (nnpMatcher.find()){
						ans = ans + " " + nnpMatcher.group(1);
					}
					else{
						i = nnps.length;
						break;
					}
					i = j;		
				}
			}
		}
		return ans;
	}

/*
 * Finds full names of albums and movies by inverted commas
 */
	
	private String findFullName(String strLine) {
		String ans = "";	
		String[] origStr = strLine.split("(``/``|\'\'/\'\')");
		if (origStr.length > 1){			
			String[] wordsArr = origStr[1].split(" ");
			for (int i = 0 ; i < wordsArr.length ; i++){
				if (wordsArr[i].length() > 0 && checkRealWord(wordsArr[i])){
					int slesh = wordsArr[i].indexOf("/");
					if (slesh >= 0)
						ans = ans + " " + wordsArr[i].substring(0,wordsArr[i].indexOf("/"));
					else 
						ans = ans + " " + wordsArr[i];
				}
			}
		}
		if (ans.length()>1)
			return ans.substring(1);
		else return "";
	}

/*
 * finds names only if it appears in the first word in the line
 */
	
	private String findNameNoSpace(String strLine) {
		String ans = "";	
		String[] nnps = strLine.split(" ");
		if (nnps.length > 1){
			String dirName = "(\\w+)/NNP";
			Pattern dirNamePattern = Pattern.compile(dirName);
			Matcher nnpMatcher;
			nnpMatcher = dirNamePattern.matcher(nnps[1]);
			if (nnpMatcher.find()){
				ans = ans + nnpMatcher.group(1) ;
				for(int j =2; j < nnps.length; j++){
					nnpMatcher = dirNamePattern.matcher(nnps[j]);
					if (nnpMatcher.find()){
						ans = ans + " " + nnpMatcher.group(1);
					}	
				}
			}
		}
		return ans;
	}

/*
 * finds a year from a sentence
 */
	
	private String findYear(String strLine) {
		String year = "(\\d\\d\\d\\d)/CD";
		Pattern yearPattern = Pattern.compile(year);
		Matcher yearMatcher = yearPattern.matcher(strLine);
		if (yearMatcher.find())
			return yearMatcher.group(1);
		return null;		
	}
/*
 * find directors name
 */
	private String findDirector(String strLine) {
		String directedBy = "directed/VBN by/IN";
		Pattern directedByPattern =Pattern.compile(directedBy);
		Matcher directedByMatcher = directedByPattern.matcher(strLine);
		if (directedByMatcher.find()){
			return  findName(strLine.substring(directedByMatcher.start()));
		}
		return null;
	}
	
//*********************************//
/*           Getters		       */
//*********************************//
	
	public Vector<String> getBornInPersons() {
		Collections.sort(bornInPersons);
		return bornInPersons;
	}

	public Vector<String> getProfPersons() {
		Collections.sort(profPersons);
		return profPersons;
	}

	public Vector<String> getAllAlbums() {
		Collections.sort(allAlbums);
		return allAlbums;
	}

	public Vector<String> getAllDirectors() {
		Collections.sort(allDirectors);
		return allDirectors;
	}

	public Vector<String> getAllCountries() {
		Collections.sort(allCountries);
		return allCountries;
	}

	public Vector<String> getAllArtists() {
		Collections.sort(allArtists);
		return allArtists;
	}

	public Vector<String> getAllDirectedMovies() {
		return allDirectedMovies;
	}
	public Table getBornIn() {
		return bornIn;
	}

	public Table getProfession() {
		return profession;
	}

	public Table getMoviesByYear() {
		return moviesByYear;
	}

	public Table getDirectors() {
		return directors;
	}

	public Table getCapitals() {
		return capitals;
	}

	public Table getAlbums() {
		return albums;
	}

	public Table getArtistsAlbums() {
		return artistsAlbums;
	}


}

